package edu.hawaii.webspider;

import java.net.ConnectException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;

/**
 * A demonstration class illustrating how to retrieve and process web pages using HttpUnit.
 * 
 * @author Jared Sunouchi
 */
public class WebSpiderExample {

  /** Contains the starting url for the crawl. */
  private final String startUrl;
  /** Current number of pages traversed from the starting url. */
  private int currentNumberOfPages = -1;
  /** Total number of pages to be traversed from the starting url. */
  private int totalNumberOfPages = 0;
  /** Contains the pages that have already been looked at. */
  private final List<String> traversedPages = new ArrayList<String>();
  /** Contains each page and how many other pages point to it. */
  private final Map<String, Integer> popularPages = new HashMap<String, Integer>();
  /** If logging is turned on. */
  private final boolean logging;
  /** The Logger used for logging. */
  private static final Logger theLogger = Logger.getLogger("edu.hawaii.webspider");
  /** Contains the pages that will be checked. */
  private final List<String> pagesToLookAt = new ArrayList<String>();
  /** Total number of links found. */
  private int totalNumberOfLinks = 0;

  /**
   * A WebSpider crawls the web and returns info.
   * 
   * @param startUrl The home url for the crawl.
   * @param logging Determines whether the user wants logging or not.
   */
  public WebSpiderExample(String startUrl, boolean logging) {
    this.startUrl = startUrl;
    this.logging = logging;
  }

  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception Thrown if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);
    return response.getLinks().length;
  }
  
  /**
   * Returns the number of links of the first defined number of pages from the startUrl.
   * 
   * @param totalNumberOfPages The total number of pages to be examined.
   * @throws Exception Thrown if problems occur retrieving the url.
   */
  public void totalLinks(int totalNumberOfPages) throws Exception {
    this.totalNumberOfPages = totalNumberOfPages;    
    
    if (logging) {
      theLogger.info("Running totalLinks.");
    }
    
    pagesToLookAt.add(startUrl);
    while ((!(pagesToLookAt.isEmpty())) && (currentNumberOfPages < this.totalNumberOfPages)) {
      String url = pagesToLookAt.remove(0); 
      if (!(traversedPages.contains(url))) {
        try {
          if (logging) {
            theLogger.info("Retrieving " + url);
          }
          collectLinks(url);
        }
        catch (ConnectException error) {
          currentNumberOfPages--;
        }
      }
    }
    
    System.out.println("The total number of links discovered while crawling the first "
                       + totalNumberOfPages + " pages accessable from " + startUrl + 
                       " is: " + totalNumberOfLinks);
  }
  
  /**
   * Collects the links for each page.
   * 
   * @param url The url of the page that the links will be collected from.
   * @throws Exception Thrown if problems occur retrieving the url.
   */
  public void collectLinks(String url) throws Exception {
    // prevent javascript errors from being thrown
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError (false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);    
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);
    
    currentNumberOfPages++;
    traversedPages.add(url);
    try {
      // create the conversation object which will maintain state for us
      WebConversation wc = new WebConversation();
      WebRequest request = new GetMethodWebRequest(url);
      WebResponse response = wc.getResponse(request);
      WebLink[] links = response.getLinks();

      if (logging) {
        theLogger.info("Found " + links.length + " links.");
      }
      for (int i = 0; i < links.length; i++) {
        pagesToLookAt.add(links[i].getRequest().getURL().toString());
        totalNumberOfLinks++;
      }
    }
    catch (RuntimeException error) {
      // NotHTMLException
      if (logging) {
        theLogger.info(error.getMessage());
      }
    }    
  }
  
  /**
   * Finds the most popular page when crawling a set amount of pages.
   * 
   * @param totalNumberOfPages The total number of pages to be examined.
   * @throws Exception Thrown if problems occur retrieving the url.
   */
  public void mostPopular(int totalNumberOfPages) throws Exception {
    this.totalNumberOfPages = totalNumberOfPages;
    String mostPopularUrl = "";
    int mostPopularPage = 0;
    
    if (logging) {
      theLogger.info("Running mostPopular.");
    }
    
    pagesToLookAt.add(startUrl);
    while ((!(pagesToLookAt.isEmpty())) && (currentNumberOfPages < this.totalNumberOfPages)) {
      String url = pagesToLookAt.remove(0); 
      if (!(traversedPages.contains(url))) {
        try {
          if (logging) {
            theLogger.info("Retrieving " + url);
          }
          findMostPopularPage(url);
        }
        catch (ConnectException error) {
          currentNumberOfPages--;
        }
      }
    }
    
    // go through each entry in the map and find the most popular page
    Set<Map.Entry<String, Integer>> pageSet = popularPages.entrySet();
    for (Map.Entry<String, Integer> page : pageSet) {
      if (page.getValue().intValue() > mostPopularPage) {
        mostPopularPage = page.getValue().intValue();
        mostPopularUrl = page.getKey();
      }
    }
    
    System.out.println("The most popular page discovered is: " + mostPopularUrl + " with " 
                       + mostPopularPage + " pages that point to it.");
  }
  
  /**
   * Alters the popularPages map to reflect the number of links that point to each URL.
   * 
   * @param url The url whose pointer links value will be updated.
   * @throws Exception Thrown if problems occur retrieving the url.
   */
  public void findMostPopularPage(String url) throws Exception {
    // prevent javascript errors from being thrown
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError (false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);    
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);
    
    currentNumberOfPages++;
    traversedPages.add(url);
    try {
      // create the conversation object which will maintain state for us
      WebConversation wc = new WebConversation();
      WebRequest request = new GetMethodWebRequest(url);
      WebResponse response = wc.getResponse(request);
      WebLink[] links = response.getLinks();

      // go through the links on the page and either add them to the list or increment the number
      // of times a page has pointed to it
      for (int i = 0; i < links.length; i++) {
        pagesToLookAt.add(links[i].getRequest().getURL().toString());
        int count = 1;
        if (popularPages.containsKey(links[i].getRequest().getURL().toString())) {
          count += popularPages.get(links[i].getRequest().getURL().toString());
        }
        popularPages.put(links[i].getRequest().getURL().toString(), count);
      }
    }
    catch (RuntimeException error) {
      // NotHTMLException
      if (logging) {
        theLogger.info(error.getMessage());
      }
    }
  }

  /**
   * Checks the validity of the passed arguments and executes the various tasks.
   * 
   * @param args Ignored.
   * @throws Exception If problems occur.
   */
  public static void main(String[] args) throws Exception {
    List<String> argsList = new ArrayList<String>();
    for (int i = 0; i < args.length; i++) {
      argsList.add(args[i]);
    }
    
    String exiting = "exiting";
    
    // check for the various tasks
    int taskCheck1 = argsList.indexOf("-totallinks");
    int taskCheck2 = argsList.indexOf("-mostpopular");
    int loggingCheck = argsList.indexOf("-logging");
    boolean logging = false;
    
    // check to see if we are logging and check if it is in the right place
    if (loggingCheck == -1) {
      if (argsList.size() != 3) {
        theLogger.log(Level.SEVERE, "Incorrect number of arguments!");
        theLogger.warning(exiting);
        return;
      }
    }
    else {
      if (loggingCheck != 3) {
        theLogger.log(Level.SEVERE, "Order of passed arguments is incorrect!");
        theLogger.warning(exiting);
        return;
      }
      logging = true;
    }
    
    // check if the task is spelled correctly and if it is the right order
    if ((taskCheck1 != 0) && (taskCheck2 != 0)) {
      if ((taskCheck1 == -1) && (taskCheck2 == -1)) {
        theLogger.log(Level.SEVERE, "Task does not exist!");
        theLogger.warning(exiting);
        return;
      }
      else {
        theLogger.log(Level.SEVERE, "Order of passed arguments is incorrect!");
        theLogger.warning(exiting);
        return;
      }
    }
    
    // if everything checks out run the task
    if (taskCheck1 > taskCheck2) {
      WebSpiderExample totalLinksExample = 
        new WebSpiderExample(argsList.get(taskCheck1 + 1), logging);
      totalLinksExample.totalLinks(Integer.parseInt(argsList.get(taskCheck1 + 2)));
    }
    else {
      WebSpiderExample mostPopularExample = 
        new WebSpiderExample(argsList.get(taskCheck2 + 1), logging);
      mostPopularExample.mostPopular(Integer.parseInt(argsList.get(taskCheck2 + 2)));
    }
  }
}
